#!/bin/sh
###############################################################################
##  Author    : xuezhoyi
##  Name      : edw_proc_launcher.sh
##  Functions : 执行过程总调
##  Purpose   : 
##  Revisions or Comments
##  VER        DATE        AUTHOR           DESCRIPTION
##---------  ----------  ---------------  ------------------------------------ 
##  1.0      2017-05-22  xuezhouyi        1. CREATED THIS SHELL.
##  2.0      2018-11-11  薛周毅           1.执行MySQL或hive过程
##                                        2.根据执行后的结果更新表【EDW_JOB_STATUS】状态
###############################################################################

. ${0%`basename ${0}`}edw_parm.sh
. ${0%`basename ${0}`}edw_func.sh

function USAGE(){
    echo "How to use this shell script!"
    echo "${V_SHELL_HOME}/edw_proc_launcher.sh HIVE 1"
    echo "${V_SHELL_HOME}/edw_proc_launcher.sh MYSQL 1"
}

function START(){
    V_QUERY="UPDATE ETL.EDW_JOB_STATUS SET START_TIME='$(NOW)' WHERE DATA_DATE='${1}' AND JOB_TYPE='${2}' AND JOB_GROUP='${3}' AND PROC_NAME='${4}'"
    ${V_RUN_DB} -q "${V_QUERY}" 
}

function P_END(){
    V_QUERY="SELECT START_TIME FROM ETL.EDW_JOB_STATUS WHERE DATA_DATE='${1}' AND JOB_TYPE='${2}' AND JOB_GROUP='${3}' AND PROC_NAME='${4}'"
    START_TIME=$(${V_RUN_DB} -q "${V_QUERY}")
    END_TIME=$(NOW)
    RUN_TIME=$(DIF_TIME "${START_TIME}" "${END_TIME}")
    V_QUERY="UPDATE ETL.EDW_JOB_STATUS SET END_TIME='${END_TIME}',RUN_TIME='${RUN_TIME}' WHERE DATA_DATE='${1}' AND JOB_TYPE='${2}' AND JOB_GROUP='${3}' AND PROC_NAME='${4}'"
    ${V_RUN_DB} -q "${V_QUERY}" 
}

function RUN(){
    V_QUERY="UPDATE ETL.EDW_JOB_STATUS SET STATUS='RUN' WHERE DATA_DATE='${1}' AND JOB_TYPE='${2}' AND JOB_GROUP='${3}' AND PROC_NAME='${4}'"
    ${V_RUN_DB} -q "${V_QUERY}" 
}

function DONE(){
    V_QUERY="UPDATE ETL.EDW_JOB_STATUS SET STATUS='DONE' WHERE DATA_DATE='${1}' AND JOB_TYPE='${2}' AND JOB_GROUP='${3}' AND PROC_NAME='${4}'"
    ${V_RUN_DB} -q "${V_QUERY}" 
}

function FAIL(){
    V_QUERY="UPDATE ETL.EDW_JOB_STATUS SET STATUS='FAIL' WHERE DATA_DATE='${1}' AND JOB_TYPE='${2}' AND JOB_GROUP='${3}' AND PROC_NAME='${4}'"
    ${V_RUN_DB} -q "${V_QUERY}"
}

# ensure the basic variavle of connection is not null
if [[ -z ${1} || -z ${2} ]];then
    USAGE
    exit 1
else
    V_JOB_TYPE=${1}
    V_PARALLEL_CNT=${2}
fi

# Get data date
V_DATA_DATE=$(grep "#V_DATA_DATE#" ${V_PARM_FILE} | awk -F'=' '{print $2}')
V_FREQUENCY=$(grep "#V_FREQUENCY#" ${V_PARM_FILE} | awk -F'=' '{print $2}')
# Define basic parm
V_SCRIPT_NAME=$(basename ${0} | cut -d'.' -f1)
V_OUT_LOGS="${V_SHELL_LOGS}/${V_DATA_DATE}/${V_SCRIPT_NAME}_${V_JOB_TYPE}_${V_FREQUENCY}_${V_PARALLEL_CNT}.log"

# Define the output flow
exec 4>&1               # screen output
exec 3>>${V_OUT_LOGS}   # script output
exec 2>&3               # error output
exec 1>&3               # standard output



BEGIN
LOGGER "INFO" "[--------------------------------------------------Thread:${V_PARALLEL_CNT}--------------------------------------------------]"
LOGGER "INFO" "Today batch date is ${V_DATA_DATE}."



# Get the group list
V_QUERY="SELECT DISTINCT JOB_GROUP FROM ETL.EDW_JOB_STATUS WHERE DATA_DATE='${V_DATA_DATE}' AND JOB_TYPE='${V_JOB_TYPE}' ORDER BY JOB_GROUP"
V_GROUP_LIST=$(${V_RUN_DB} -q "${V_QUERY}") || exit 2
for GROUP in ${V_GROUP_LIST};do
    LOGGER "INFO" "[--------------------------------------------------Start to group:${GROUP}--------------------------------------------------]"
    V_QUERY="SELECT DISTINCT PROC_NAME FROM ETL.EDW_JOB_STATUS WHERE DATA_DATE='${V_DATA_DATE}' AND JOB_TYPE='${V_JOB_TYPE}' AND JOB_GROUP='${GROUP}' AND MOD(SEQ_NO,${V_PARALLEL_PROC_NO})='${V_PARALLEL_CNT}' AND STATUS IS NULL"
    V_PROC_LIST=$(${V_RUN_DB} -q "${V_QUERY}") || exit 3
    for PROC in ${V_PROC_LIST};do
        START ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 4
        # Hive or mysql
        case ${V_JOB_TYPE} in
            HIVE)
                # Check the proc file if not exists
                if [[ ! -f "${V_SHELL_PROC}/${PROC}" ]];then
                    FAIL ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 5
                    LOGGER "ERROR" "Can not find the procedure file:${V_SHELL_PROC}/${PROC}"
                    continue
                else
                    LOGGER "INFO" "******************************Run ${V_JOB_TYPE} procedure:${PROC}******************************"
                fi
                
                # Run procedure
                RUN ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 6
                
                # create the folder if not exists
                if [[ ! -d "${V_SHELL_PROC}/tmp" ]];then
                    mkdir -p "${V_SHELL_PROC}/tmp" || exit 7
                fi
                
                # Create a copy file
                cp "${V_SHELL_PROC}/${PROC}" "${V_SHELL_PROC}/tmp/${PROC}" 
                while read line;do
                    V_KEY=$(echo ${line} | awk -F'=' '{print $1}')
                    V_VAL=$(echo ${line} | awk -F'=' '{print $2}')
                    sed -i "s/${V_KEY}/${V_VAL}/g" "${V_SHELL_PROC}/tmp/${PROC}" || exit 8
                done < "${V_PARM_FILE}"
                
                # Run hive sql file
                ${V_RUN_HIVQL} -f ${V_SHELL_PROC}/tmp/${PROC} 
                if [[ $? -eq 0 ]];then
                    DONE ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 9
                    LOGGER "INFO" "Success to run procedure[${V_SHELL_PROC}/${PROC}]"
                else
                    FAIL ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 10
                    LOGGER "ERROR" "Fail to run procedure[${PROC}]."
                    LOGGER "DEBUG" "${V_RUN_HIVQL} -f ${V_SHELL_PROC}/tmp/${PROC}"
                    exit 10
                fi
                P_END ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 11
            ;;
            MYSQL)
                LOGGER "INFO" "***RUN ${V_JOB_TYPE} procedure:${PROC}***"
                RUN ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 12
                V_QUERY="CALL ${PROC}('${V_DATA_DATE}',@RC);SELECT @RC"
                V_RC=$(${V_RUN_DB} -q "${V_QUERY}") || exit 13
                if [[ ${V_RC} -eq 0 ]];then
                    DONE ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 14
                    LOGGER "INFO" "Success to run procedure[${PROC}]"
                else
                    FAIL ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 15
                    LOGGER "ERROR" "Fail to run procedure[${PROC}]."
                    LOGGER "DEBUG" "${V_QUERY}"
                    exit 15
                fi
                P_END ${V_DATA_DATE} ${V_JOB_TYPE} ${GROUP} ${PROC} || exit 16
            ;;
            KYLIN)
                echo "麒麟待开发"
            ;;
        esac
      echo "JOB_DONE=1" >> ${PROC_SPEED_FILE}
    done
    
    # Waiting for all jobs completed
    LOGGER "WARNING" "等待${GROUP}组其他线程任务结束..."
    while true;do
        V_QUERY="SELECT COALESCE(COUNT(1),0) FROM ETL.EDW_JOB_STATUS WHERE DATA_DATE='${V_DATA_DATE}' AND JOB_TYPE='${V_JOB_TYPE}' AND JOB_GROUP='${GROUP}' AND (STATUS IS NULL OR STATUS='RUN')"
        V_DONE_CNT=$(${V_RUN_DB} -q "${V_QUERY}") || exit 17
        if [[ ${V_DONE_CNT} -eq 0 ]];then
            break
        fi

        V_QUERY="SELECT COALESCE(COUNT(1),0) FROM ETL.EDW_JOB_STATUS WHERE DATA_DATE='${V_DATA_DATE}' AND JOB_TYPE='${V_JOB_TYPE}' AND JOB_GROUP='${GROUP}' AND STATUS='FAIL'"
        V_FAIL_CNT=$(${V_RUN_DB} -q "${V_QUERY}") || exit 18
        if [[ ${V_FAIL_CNT} -ne 0 ]];then
            LOGGER "WARN" "${GROUP} 组任务执行失败,任务退出！详情请查询日志表 EDW_JOB_STATUS."
            exit 0
        fi
    done
done

END
exit 0
